# -*- coding:utf-8 -*-

import requests
from lxml import etree
from fake_useragent import UserAgent
from http import cookiejar
import json
from datetime import datetime
import time
import re
import csv
from pymongo import MongoClient
from proxy_auth.proxy_auth_mogu import *

requests.packages.urllib3.disable_warnings()  # 忽略HTTPS安全警告

"""
中国注册会计师
http://cmispub.cicpa.org.cn/
"""


class CAP():
    def __init__(self):
        #声明一个CookieJar对象实例来保存cookie
        # requests.utils.dict_from_cookiejar(html.cookies)  # 获取cookies转dict
        self.cookie = cookiejar.CookieJar()
        ua = UserAgent(use_cache_server=False)  # 禁用服务器缓存
        self.headers = {
            # 'User-Agent': ua.random,
            # 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko',
            # 'Accept': 'text/html, application/xhtml+xml, image/jxr, */*',
            # 'Accept-Encoding': 'gzip, deflate',
            # 'Accept-Language': 'zh-CN',
            # 'Connection': 'Keep-Alive',
            # 'Host': 'cmispub.cicpa.org.cn',
            # 'Cookie': 'JSESSIONID=92F1394BE6A7C01A0657DBC21AD17390; cookiee=20111116'
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
            'Accept-Language': 'en',
            "user-agent": "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko"


        }
        self.url = "http://cmispub.cicpa.org.cn/cicpa2_web/OfficeIndexAction.do"


    # 获取会计事务所信息
    def get_contents(self, cursor):
        postData = {
            'pageSize':'15',
            'pageNum':cursor,
            'method':'indexQuery',
            'queryType':1,
            'isStock':'00',
            'ascGuid':'00',  # 所在地区 00 全部
            'offName':'',  # 会计师事务所名称
            'offAllcode':'',  # 会计师事务所证书编号
            'personNum':''
        }
        url = f"http://cmispub.cicpa.org.cn/cicpa2_web/OfficeIndexAction.do"
        html = HttpUtils.do_request("POST", self.url, self.headers, postData)
        # html = get_request_proxy("POST", url, self.headers, postData)
        root = etree.HTML(html.text)
        count = 0
        for r in root.xpath('//table[@id="tabDetail"]/tr'):
            if count > 0:

                dict_data = dict()
                dict_data['事务所代码'] = r.xpath('./td[2]/text()')[0]  # 事务所代码
                dict_data['事务所名称'] = r.xpath('./td[3]/a/text()')[0]  # 事务所名称
                id = r.xpath('./td[3]/a/@href')[0].replace("javascript:viewDetail(", "").replace(")", "").split(",")[0].replace("'", "")
                id_1 = r.xpath('./td[3]/a/@href')[0].replace("javascript:viewDetail(", "").replace(")", "").split(",")[1].replace("'", "")
                url_child = f"http://cmispub.cicpa.org.cn/cicpa2_web/09/{id}.shtml"  # 不会跳转，但会报错
                url_child_1 = f"http://cmispub.cicpa.org.cn/cicpa2_web/002/{id},{id_1}/7.shtml"  # 会跳转
                dict_data['通讯地址'] = r.xpath('./td[4]/text()')[0]  # 通讯地址
                dict_data['联系人'] = "".join(r.xpath('./td[5]/text()'))  # 联系人
                dict_data['联系电话'] = "".join(r.xpath('./td[6]/text()'))  # 联系电话
                # print(str(dict_data))
                # self.insertItem("中国注册会计师_事务所基本信息", dict_data)
                # 获取详细信息
                self.get_detail(url_child, url_child_1, dict_data['事务所名称'])  # url_child不会跳转，先使用代理，
                # 获取注册会计师
                self.get_kj(id, 1, dict_data['事务所名称'])
                # # 获取从业人员
                self.get_cy(id, 1, dict_data['事务所名称'])

            count += 1

    # 获取事务所详细信息
    def get_detail(self, url, url_1, company):
        print(f"============获取【{company}】详细信息=============")
        html = HttpUtils.do_request("GET", url_1, self.headers, "")
        # html = get_request_proxy("GET", url, self.headers, "")
        html.encoding = "GB2312"
        root = etree.HTML(html.text)
        # # 先使用代理，如果代理不通。用本地
        # if len(root.xpath('//table[@id="detailtb"]/tr[3]/td[2]/text()')) == 0:
        #     html = HttpUtils.do_request("GET", url_1, self.headers, "")
        #     html.encoding = "GB2312"
        #     root = etree.HTML(html.text)
        #     if len(root.xpath('//table[@id="detailtb"]/tr[3]/td[2]/text()')) > 0:
        #         if root.xpath('//table[@id="detailtb"]/tr[3]/td[2]/text()')[0] == "":
        #             self.get_detail(url, url_1, dict_data)
        #     else:
        #         self.get_detail(url, url_1, dict_data)
        dict_data = dict()
        # 提取信息
        for r in root.xpath('//table[@id="detailtb"]/tr'):
            if len(r.xpath('./td[@class="tdl"]')) > 0:
                td_num = 0
                for c in r.xpath('./td[@class="tdl"]'):
                    tag_name = r.xpath('./td[@class="tdl"]/text()')[td_num].replace("\n", "").replace("\t", "").strip()
                    tag_value = r.xpath('./td[@class="data_tb_content"]/text()')[td_num].replace("\n", "").replace("\t", "").strip()
                    dict_data[tag_name] = tag_value
                    td_num += 1

        # # print(str(dict_data))
        self.insertItem("中国注册会计师_事务所详细信息", dict_data)


    # 获取注册会计师
    def get_kj(self, kj_id, current, company):
        postData = {
            'method': 'getPersons',
            'offGuid': kj_id,
            'pageNum': current,
            'pageSize': '',
            'title':company
        }
        url = "http://cmispub.cicpa.org.cn/cicpa2_web/OfficeIndexAction.do"
        html = HttpUtils.do_request("GET", url, self.headers, postData)
        # html = get_request_proxy("GET", url, self.headers, postData)

        html.encoding = "GB2312"
        root = etree.HTML(html.text)
        # 获取总页数
        page_info = ''.join(root.xpath('.//div[@id="leftctrl"]//text()'))
        total_page = re.search(' 共(.+?)页', page_info)
        if total_page is not None:
            total_page = re.search(' 共(.+?)页', page_info).group(1).strip()
            count = 0
            print(f"============获取【{company}】【注册会计师】总页数【{total_page}】当前页【{current}】=============")
            for r in root.xpath('//table[@id="pertable"]/tr'):
                if count > 0:
                    dict_data = dict()
                    dict_data['姓名'] = r.xpath('./td[2]/a/text()')[0].replace("\r", "").replace("\n", "").replace("\t", "").strip()
                    person_id = r.xpath('./td[2]/a/@onclick')[0].replace("getPerDetails(", "").replace(")", "").split(",")[0].replace("'", "")
                    dict_data['人员编号'] = r.xpath('./td[3]/text()')[0].replace("\r", "").replace("\n", "").replace("\t", "").strip()
                    dict_data['性别'] = r.xpath('./td[4]/text()')[0].replace("\r", "").replace("\n", "").replace("\t", "").strip()
                    dict_data['出生日期'] = r.xpath('./td[5]/text()')[0].replace("\r", "").replace("\n", "").replace("\t", "").strip()
                    dict_data['全科合格证号（或者考核批准文号）'] = r.xpath('./td[6]/text()')[0].replace("\r", "").replace("\n", "").replace("\t", "").strip()
                    # print(str(dict_data))
                    # 获取注册会计师详细信息
                    self.get_kj_detail(person_id)
                count += 1

            # 翻页
            if current < int(total_page):
                self.get_kj(kj_id, current + 1, company)


    # 获取从业人员
    def get_cy(self, kj_id, current, company):
        postData = {
            'method': 'getEmployeeList',
            'offGuid': kj_id,
            'pageNum': current,
            'pageSize': '',
            'title':company
        }
        url = "http://cmispub.cicpa.org.cn/cicpa2_web/OfficeIndexAction.do"
        html = HttpUtils.do_request("GET", url, self.headers, postData)
        # html = get_request_proxy("GET", url, self.headers, postData)
        html.encoding = "GB2312"
        root = etree.HTML(html.text)
        # 获取总页数
        page_info = ''.join(root.xpath('.//div[@id="leftctrl"]//text()'))
        total_page = re.search(' 共(.+?)页', page_info)
        if total_page is not None:
            total_page = re.search(' 共(.+?)页', page_info).group(1).strip()
            count = 0
            print(f"============获取【{company}】【从业人员】总页数【{total_page}】当前页【{current}】=============")
            for r in root.xpath('//table[@id="nwp"]/tr'):
                if count > 1:
                    dict_data = dict()
                    dict_data['事务所名称'] = company
                    dict_data['姓名'] = r.xpath('./td[2]/text()')[0].replace("\r", "").replace("\n", "").replace("\t", "").strip()
                    dict_data['性别'] = r.xpath('./td[3]/text()')[0].replace("\r", "").replace("\n", "").replace("\t", "").strip()
                    dict_data['进所时间'] = r.xpath('./td[4]/text()')[0].replace("\r", "").replace("\n", "").replace("\t", "").strip()
                    dict_data['是否签合同'] = r.xpath('./td[5]/text()')[0].replace("\r", "").replace("\n", "").replace("\t", "").strip()
                    dict_data['是否参加社保'] = r.xpath('./td[6]/text()')[0].replace("\r", "").replace("\n", "").replace("\t", "").strip()
                    dict_data['是否党员'] = r.xpath('./td[7]/text()')[0].replace("\r", "").replace("\n", "").replace("\t", "").strip()
                    # print(str(dict_data))
                    self.insertItem("中国注册会计师_从业人员信息", dict_data)

                count += 1

            # 翻页
            if current < int(total_page):
                self.get_cy(kj_id, current + 1, company)

    # 获取注册会计师详细信息
    def get_kj_detail(self, id):
        url = f"http://cmispub.cicpa.org.cn/cicpa2_web/07/{id}.shtml"
        html = HttpUtils.do_request("GET", url, self.headers, "")
        # html = get_request_proxy("GET", url, self.headers, "")
        # if html.status_code == 404 or html is None:
        #     url = f"http://cmispub.cicpa.org.cn/cicpa2_web/public/query/sws/p/{id}.shtml"
        #     html = HttpUtils.do_request("GET", url, self.headers, "")
        #     html.encoding = "GB2312"
        #     root = etree.HTML(html.text)
        #     if len(root.xpath('//table[@id="detailtb"]/tr[3]/td[2]/text()')) == 0:
        #         self.get_kj_detail(id)
        if html.status_code == 404:
            print(f"====404:{id}====")
            return
        html.encoding = "GB2312"
        root = etree.HTML(html.text)
        dict_data = dict()
        # 提取信息
        for r in root.xpath('//table[@id="detailtb"]/tr'):
            if len(r.xpath('./td[@class="tdl"]')) > 0:
                td_num = 0
                for c in r.xpath('./td[@class="tdl"]'):
                    tag_name = r.xpath('./td[@class="tdl"]/text()')[td_num].replace("\n", "").replace("\t", "").strip()
                    tag_value = r.xpath('./td[@class="data_tb_content"]/text()')[td_num].replace("\n", "").replace("\t", "").strip()
                    dict_data[tag_name] = tag_value
                    td_num += 1

        self.insertItem("中国注册会计师_注册会计师详细信息", dict_data)

    def insertItem(self, tableName, data):
        my_set = db[tableName]
        my_set.insert_one(data)


if __name__ == '__main__':
    conn = MongoClient('127.0.0.1', 27017)
    db = conn["Test"]
    cap = CAP()
    for i in range(177, 636):
        print(f"============第【{i}】页=============")
        cap.get_contents(i)
    # cap.get_kj('0000010F849689DD9F090908201C39DB', 1, "北京市华颂会计师事务所有限公司")
